{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use SARSA($\\lambda$) to Play MoutainCar-v0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import sys\n",
    "import logging\n",
    "import itertools\n",
    "\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "import gym\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "logging.basicConfig(level=logging.DEBUG,\n",
    "        format='%(asctime)s [%(levelname)s] %(message)s',\n",
    "        stream=sys.stdout, datefmt='%H:%M:%S')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15:04:18 [INFO] env: <MountainCarEnv<MountainCar-v0>>\n",
      "15:04:18 [INFO] action_space: Discrete(3)\n",
      "15:04:18 [INFO] observation_space: Box(-1.2000000476837158, 0.6000000238418579, (2,), float32)\n",
      "15:04:18 [INFO] reward_range: (-inf, inf)\n",
      "15:04:18 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 30}\n",
      "15:04:18 [INFO] _max_episode_steps: 200\n",
      "15:04:18 [INFO] _elapsed_steps: None\n",
      "15:04:18 [INFO] id: MountainCar-v0\n",
      "15:04:18 [INFO] entry_point: gym.envs.classic_control:MountainCarEnv\n",
      "15:04:18 [INFO] reward_threshold: -110.0\n",
      "15:04:18 [INFO] nondeterministic: False\n",
      "15:04:18 [INFO] max_episode_steps: 200\n",
      "15:04:18 [INFO] _kwargs: {}\n",
      "15:04:18 [INFO] _env_name: MountainCar\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('MountainCar-v0')\n",
    "env.seed(0)\n",
    "for key in vars(env):\n",
    "    logging.info('%s: %s', key, vars(env)[key])\n",
    "for key in vars(env.spec):\n",
    "    logging.info('%s: %s', key, vars(env.spec)[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TileCoder:\n",
    "    def __init__(self, layer_count, feature_count):\n",
    "        self.layer_count = layer_count\n",
    "        self.feature_count = feature_count\n",
    "        self.codebook = {}\n",
    "\n",
    "    def get_feature(self, codeword):\n",
    "        if codeword in self.codebook:\n",
    "            return self.codebook[codeword]\n",
    "        count = len(self.codebook)\n",
    "        if count >= self.feature_count: # resolve conflicts\n",
    "            return hash(codeword) % self.feature_count\n",
    "        self.codebook[codeword] = count\n",
    "        return count\n",
    "\n",
    "    def __call__(self, floats=(), ints=()):\n",
    "        dim = len(floats)\n",
    "        scaled_floats = tuple(f * (self.layer_count ** 2) for f in floats)\n",
    "        features = []\n",
    "        for layer in range(self.layer_count):\n",
    "            codeword = (layer,) + tuple(\n",
    "                    int((f + (1 + dim * i) * layer) / self.layer_count)\n",
    "                    for i, f in enumerate(scaled_floats)) + ints\n",
    "            feature = self.get_feature(codeword)\n",
    "            features.append(feature)\n",
    "        return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SARSALambdaAgent:\n",
    "    def __init__(self, env):\n",
    "        self.action_n = env.action_space.n\n",
    "        self.obs_low = env.observation_space.low\n",
    "        self.obs_scale = env.observation_space.high - \\\n",
    "                env.observation_space.low\n",
    "        self.encoder = TileCoder(8, 1896)\n",
    "        self.w = np.zeros(self.encoder.feature_count)\n",
    "        self.gamma = 1.\n",
    "        self.learning_rate = 0.03\n",
    "\n",
    "    def encode(self, observation, action):\n",
    "        states = tuple((observation - self.obs_low) / self.obs_scale)\n",
    "        actions = (action,)\n",
    "        return self.encoder(states, actions)\n",
    "\n",
    "    def get_q(self, observation, action): # action value\n",
    "        features = self.encode(observation, action)\n",
    "        return self.w[features].sum()\n",
    "\n",
    "    def reset(self, mode=None):\n",
    "        self.mode = mode\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory = []\n",
    "            self.z = np.zeros(self.encoder.feature_count) # eligibility trace\n",
    "\n",
    "    def step(self, observation, reward, done):\n",
    "        if self.mode == 'train' and np.random.rand() < 0.001:\n",
    "            action = np.random.randint(self.action_n)\n",
    "        else:\n",
    "            qs = [self.get_q(observation, action) for action in\n",
    "                    range(self.action_n)]\n",
    "            action = np.argmax(qs)\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory += [observation, reward, done, action]\n",
    "            if len(self.trajectory) >= 8:\n",
    "                self.learn()\n",
    "        return action\n",
    "\n",
    "    def close(self):\n",
    "        pass\n",
    "\n",
    "    def learn(self):\n",
    "        observation, _, _, action, next_observation, reward, done, \\\n",
    "                next_action = self.trajectory[-8:]\n",
    "        target = reward + (1. - done) * self.gamma * \\\n",
    "                self.get_q(next_observation, next_action)\n",
    "        td_error = target - self.get_q(observation, action)\n",
    "\n",
    "        # update replace trace\n",
    "        self.z *= (self.gamma * 0.9) # 0.9 is the lambda value\n",
    "        features = self.encode(observation, action)\n",
    "        self.z[features] = 1.\n",
    "\n",
    "        self.w += (self.learning_rate * td_error * self.z)\n",
    "\n",
    "\n",
    "agent = SARSALambdaAgent(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15:04:18 [INFO] ==== train ====\n",
      "15:04:19 [DEBUG] train episode 0: reward = -200.00, steps = 200\n",
      "15:04:19 [DEBUG] train episode 1: reward = -200.00, steps = 200\n",
      "15:04:19 [DEBUG] train episode 2: reward = -200.00, steps = 200\n",
      "15:04:19 [DEBUG] train episode 3: reward = -200.00, steps = 200\n",
      "15:04:19 [DEBUG] train episode 4: reward = -200.00, steps = 200\n",
      "15:04:19 [DEBUG] train episode 5: reward = -200.00, steps = 200\n",
      "15:04:20 [DEBUG] train episode 6: reward = -200.00, steps = 200\n",
      "15:04:20 [DEBUG] train episode 7: reward = -200.00, steps = 200\n",
      "15:04:20 [DEBUG] train episode 8: reward = -200.00, steps = 200\n",
      "15:04:20 [DEBUG] train episode 9: reward = -200.00, steps = 200\n",
      "15:04:20 [DEBUG] train episode 10: reward = -200.00, steps = 200\n",
      "15:04:20 [DEBUG] train episode 11: reward = -200.00, steps = 200\n",
      "15:04:21 [DEBUG] train episode 12: reward = -200.00, steps = 200\n",
      "15:04:21 [DEBUG] train episode 13: reward = -200.00, steps = 200\n",
      "15:04:21 [DEBUG] train episode 14: reward = -200.00, steps = 200\n",
      "15:04:21 [DEBUG] train episode 15: reward = -200.00, steps = 200\n",
      "15:04:21 [DEBUG] train episode 16: reward = -200.00, steps = 200\n",
      "15:04:21 [DEBUG] train episode 17: reward = -200.00, steps = 200\n",
      "15:04:22 [DEBUG] train episode 18: reward = -200.00, steps = 200\n",
      "15:04:22 [DEBUG] train episode 19: reward = -200.00, steps = 200\n",
      "15:04:22 [DEBUG] train episode 20: reward = -200.00, steps = 200\n",
      "15:04:22 [DEBUG] train episode 21: reward = -200.00, steps = 200\n",
      "15:04:22 [DEBUG] train episode 22: reward = -200.00, steps = 200\n",
      "15:04:22 [DEBUG] train episode 23: reward = -200.00, steps = 200\n",
      "15:04:23 [DEBUG] train episode 24: reward = -200.00, steps = 200\n",
      "15:04:23 [DEBUG] train episode 25: reward = -200.00, steps = 200\n",
      "15:04:23 [DEBUG] train episode 26: reward = -200.00, steps = 200\n",
      "15:04:23 [DEBUG] train episode 27: reward = -200.00, steps = 200\n",
      "15:04:23 [DEBUG] train episode 28: reward = -200.00, steps = 200\n",
      "15:04:23 [DEBUG] train episode 29: reward = -200.00, steps = 200\n",
      "15:04:23 [DEBUG] train episode 30: reward = -200.00, steps = 200\n",
      "15:04:24 [DEBUG] train episode 31: reward = -200.00, steps = 200\n",
      "15:04:24 [DEBUG] train episode 32: reward = -200.00, steps = 200\n",
      "15:04:24 [DEBUG] train episode 33: reward = -176.00, steps = 176\n",
      "15:04:24 [DEBUG] train episode 34: reward = -200.00, steps = 200\n",
      "15:04:24 [DEBUG] train episode 35: reward = -200.00, steps = 200\n",
      "15:04:24 [DEBUG] train episode 36: reward = -177.00, steps = 177\n",
      "15:04:24 [DEBUG] train episode 37: reward = -200.00, steps = 200\n",
      "15:04:25 [DEBUG] train episode 38: reward = -200.00, steps = 200\n",
      "15:04:25 [DEBUG] train episode 39: reward = -200.00, steps = 200\n",
      "15:04:25 [DEBUG] train episode 40: reward = -200.00, steps = 200\n",
      "15:04:25 [DEBUG] train episode 41: reward = -200.00, steps = 200\n",
      "15:04:25 [DEBUG] train episode 42: reward = -200.00, steps = 200\n",
      "15:04:25 [DEBUG] train episode 43: reward = -200.00, steps = 200\n",
      "15:04:26 [DEBUG] train episode 44: reward = -183.00, steps = 183\n",
      "15:04:26 [DEBUG] train episode 45: reward = -200.00, steps = 200\n",
      "15:04:26 [DEBUG] train episode 46: reward = -187.00, steps = 187\n",
      "15:04:26 [DEBUG] train episode 47: reward = -200.00, steps = 200\n",
      "15:04:26 [DEBUG] train episode 48: reward = -177.00, steps = 177\n",
      "15:04:26 [DEBUG] train episode 49: reward = -87.00, steps = 87\n",
      "15:04:26 [DEBUG] train episode 50: reward = -128.00, steps = 128\n",
      "15:04:26 [DEBUG] train episode 51: reward = -110.00, steps = 110\n",
      "15:04:27 [DEBUG] train episode 52: reward = -86.00, steps = 86\n",
      "15:04:27 [DEBUG] train episode 53: reward = -172.00, steps = 172\n",
      "15:04:27 [DEBUG] train episode 54: reward = -151.00, steps = 151\n",
      "15:04:27 [DEBUG] train episode 55: reward = -108.00, steps = 108\n",
      "15:04:27 [DEBUG] train episode 56: reward = -187.00, steps = 187\n",
      "15:04:27 [DEBUG] train episode 57: reward = -92.00, steps = 92\n",
      "15:04:27 [DEBUG] train episode 58: reward = -85.00, steps = 85\n",
      "15:04:27 [DEBUG] train episode 59: reward = -144.00, steps = 144\n",
      "15:04:27 [DEBUG] train episode 60: reward = -87.00, steps = 87\n",
      "15:04:28 [DEBUG] train episode 61: reward = -112.00, steps = 112\n",
      "15:04:28 [DEBUG] train episode 62: reward = -113.00, steps = 113\n",
      "15:04:28 [DEBUG] train episode 63: reward = -165.00, steps = 165\n",
      "15:04:28 [DEBUG] train episode 64: reward = -158.00, steps = 158\n",
      "15:04:28 [DEBUG] train episode 65: reward = -114.00, steps = 114\n",
      "15:04:28 [DEBUG] train episode 66: reward = -100.00, steps = 100\n",
      "15:04:28 [DEBUG] train episode 67: reward = -157.00, steps = 157\n",
      "15:04:28 [DEBUG] train episode 68: reward = -169.00, steps = 169\n",
      "15:04:28 [DEBUG] train episode 69: reward = -160.00, steps = 160\n",
      "15:04:29 [DEBUG] train episode 70: reward = -194.00, steps = 194\n",
      "15:04:29 [DEBUG] train episode 71: reward = -158.00, steps = 158\n",
      "15:04:29 [DEBUG] train episode 72: reward = -118.00, steps = 118\n",
      "15:04:29 [DEBUG] train episode 73: reward = -158.00, steps = 158\n",
      "15:04:29 [DEBUG] train episode 74: reward = -159.00, steps = 159\n",
      "15:04:29 [DEBUG] train episode 75: reward = -157.00, steps = 157\n",
      "15:04:30 [DEBUG] train episode 76: reward = -148.00, steps = 148\n",
      "15:04:30 [DEBUG] train episode 77: reward = -174.00, steps = 174\n",
      "15:04:30 [DEBUG] train episode 78: reward = -158.00, steps = 158\n",
      "15:04:30 [DEBUG] train episode 79: reward = -127.00, steps = 127\n",
      "15:04:30 [DEBUG] train episode 80: reward = -118.00, steps = 118\n",
      "15:04:30 [DEBUG] train episode 81: reward = -143.00, steps = 143\n",
      "15:04:30 [DEBUG] train episode 82: reward = -106.00, steps = 106\n",
      "15:04:30 [DEBUG] train episode 83: reward = -198.00, steps = 198\n",
      "15:04:31 [DEBUG] train episode 84: reward = -139.00, steps = 139\n",
      "15:04:31 [DEBUG] train episode 85: reward = -179.00, steps = 179\n",
      "15:04:31 [DEBUG] train episode 86: reward = -113.00, steps = 113\n",
      "15:04:31 [DEBUG] train episode 87: reward = -143.00, steps = 143\n",
      "15:04:31 [DEBUG] train episode 88: reward = -113.00, steps = 113\n",
      "15:04:31 [DEBUG] train episode 89: reward = -109.00, steps = 109\n",
      "15:04:31 [DEBUG] train episode 90: reward = -108.00, steps = 108\n",
      "15:04:31 [DEBUG] train episode 91: reward = -109.00, steps = 109\n",
      "15:04:31 [DEBUG] train episode 92: reward = -108.00, steps = 108\n",
      "15:04:31 [DEBUG] train episode 93: reward = -107.00, steps = 107\n",
      "15:04:32 [DEBUG] train episode 94: reward = -154.00, steps = 154\n",
      "15:04:32 [DEBUG] train episode 95: reward = -153.00, steps = 153\n",
      "15:04:32 [DEBUG] train episode 96: reward = -106.00, steps = 106\n",
      "15:04:32 [DEBUG] train episode 97: reward = -141.00, steps = 141\n",
      "15:04:32 [DEBUG] train episode 98: reward = -105.00, steps = 105\n",
      "15:04:32 [DEBUG] train episode 99: reward = -105.00, steps = 105\n",
      "15:04:32 [DEBUG] train episode 100: reward = -105.00, steps = 105\n",
      "15:04:32 [DEBUG] train episode 101: reward = -105.00, steps = 105\n",
      "15:04:32 [DEBUG] train episode 102: reward = -143.00, steps = 143\n",
      "15:04:33 [DEBUG] train episode 103: reward = -145.00, steps = 145\n",
      "15:04:33 [DEBUG] train episode 104: reward = -106.00, steps = 106\n",
      "15:04:33 [DEBUG] train episode 105: reward = -106.00, steps = 106\n",
      "15:04:33 [DEBUG] train episode 106: reward = -145.00, steps = 145\n",
      "15:04:33 [DEBUG] train episode 107: reward = -105.00, steps = 105\n",
      "15:04:33 [DEBUG] train episode 108: reward = -105.00, steps = 105\n",
      "15:04:33 [DEBUG] train episode 109: reward = -168.00, steps = 168\n",
      "15:04:33 [DEBUG] train episode 110: reward = -106.00, steps = 106\n",
      "15:04:33 [DEBUG] train episode 111: reward = -105.00, steps = 105\n",
      "15:04:33 [DEBUG] train episode 112: reward = -105.00, steps = 105\n",
      "15:04:34 [DEBUG] train episode 113: reward = -146.00, steps = 146\n",
      "15:04:34 [DEBUG] train episode 114: reward = -105.00, steps = 105\n",
      "15:04:34 [DEBUG] train episode 115: reward = -145.00, steps = 145\n",
      "15:04:34 [DEBUG] train episode 116: reward = -105.00, steps = 105\n",
      "15:04:34 [DEBUG] train episode 117: reward = -106.00, steps = 106\n",
      "15:04:34 [DEBUG] train episode 118: reward = -166.00, steps = 166\n",
      "15:04:34 [DEBUG] train episode 119: reward = -154.00, steps = 154\n",
      "15:04:34 [DEBUG] train episode 120: reward = -105.00, steps = 105\n",
      "15:04:34 [DEBUG] train episode 121: reward = -105.00, steps = 105\n",
      "15:04:34 [DEBUG] train episode 122: reward = -107.00, steps = 107\n",
      "15:04:34 [DEBUG] train episode 123: reward = -106.00, steps = 106\n",
      "15:04:35 [DEBUG] train episode 124: reward = -103.00, steps = 103\n",
      "15:04:35 [DEBUG] train episode 125: reward = -85.00, steps = 85\n",
      "15:04:35 [DEBUG] train episode 126: reward = -105.00, steps = 105\n",
      "15:04:35 [DEBUG] train episode 127: reward = -104.00, steps = 104\n",
      "15:04:35 [DEBUG] train episode 128: reward = -105.00, steps = 105\n",
      "15:04:35 [INFO] ==== test ====\n",
      "15:04:35 [DEBUG] test episode 0: reward = -106.00, steps = 106\n",
      "15:04:35 [DEBUG] test episode 1: reward = -104.00, steps = 104\n",
      "15:04:35 [DEBUG] test episode 2: reward = -106.00, steps = 106\n",
      "15:04:35 [DEBUG] test episode 3: reward = -105.00, steps = 105\n",
      "15:04:35 [DEBUG] test episode 4: reward = -106.00, steps = 106\n",
      "15:04:35 [DEBUG] test episode 5: reward = -106.00, steps = 106\n",
      "15:04:35 [DEBUG] test episode 6: reward = -111.00, steps = 111\n",
      "15:04:35 [DEBUG] test episode 7: reward = -106.00, steps = 106\n",
      "15:04:35 [DEBUG] test episode 8: reward = -105.00, steps = 105\n",
      "15:04:35 [DEBUG] test episode 9: reward = -106.00, steps = 106\n",
      "15:04:35 [DEBUG] test episode 10: reward = -105.00, steps = 105\n",
      "15:04:35 [DEBUG] test episode 11: reward = -200.00, steps = 200\n",
      "15:04:35 [DEBUG] test episode 12: reward = -196.00, steps = 196\n",
      "15:04:36 [DEBUG] test episode 13: reward = -104.00, steps = 104\n",
      "15:04:36 [DEBUG] test episode 14: reward = -106.00, steps = 106\n",
      "15:04:36 [DEBUG] test episode 15: reward = -107.00, steps = 107\n",
      "15:04:36 [DEBUG] test episode 16: reward = -106.00, steps = 106\n",
      "15:04:36 [DEBUG] test episode 17: reward = -104.00, steps = 104\n",
      "15:04:36 [DEBUG] test episode 18: reward = -90.00, steps = 90\n",
      "15:04:36 [DEBUG] test episode 19: reward = -90.00, steps = 90\n",
      "15:04:36 [DEBUG] test episode 20: reward = -88.00, steps = 88\n",
      "15:04:36 [DEBUG] test episode 21: reward = -104.00, steps = 104\n",
      "15:04:36 [DEBUG] test episode 22: reward = -104.00, steps = 104\n",
      "15:04:36 [DEBUG] test episode 23: reward = -106.00, steps = 106\n",
      "15:04:36 [DEBUG] test episode 24: reward = -106.00, steps = 106\n",
      "15:04:36 [DEBUG] test episode 25: reward = -104.00, steps = 104\n",
      "15:04:36 [DEBUG] test episode 26: reward = -104.00, steps = 104\n",
      "15:04:36 [DEBUG] test episode 27: reward = -102.00, steps = 102\n",
      "15:04:36 [DEBUG] test episode 28: reward = -104.00, steps = 104\n",
      "15:04:36 [DEBUG] test episode 29: reward = -104.00, steps = 104\n",
      "15:04:36 [DEBUG] test episode 30: reward = -105.00, steps = 105\n",
      "15:04:36 [DEBUG] test episode 31: reward = -106.00, steps = 106\n",
      "15:04:36 [DEBUG] test episode 32: reward = -85.00, steps = 85\n",
      "15:04:36 [DEBUG] test episode 33: reward = -104.00, steps = 104\n",
      "15:04:36 [DEBUG] test episode 34: reward = -89.00, steps = 89\n",
      "15:04:37 [DEBUG] test episode 35: reward = -106.00, steps = 106\n",
      "15:04:37 [DEBUG] test episode 36: reward = -123.00, steps = 123\n",
      "15:04:37 [DEBUG] test episode 37: reward = -105.00, steps = 105\n",
      "15:04:37 [DEBUG] test episode 38: reward = -90.00, steps = 90\n",
      "15:04:37 [DEBUG] test episode 39: reward = -124.00, steps = 124\n",
      "15:04:37 [DEBUG] test episode 40: reward = -105.00, steps = 105\n",
      "15:04:37 [DEBUG] test episode 41: reward = -104.00, steps = 104\n",
      "15:04:37 [DEBUG] test episode 42: reward = -106.00, steps = 106\n",
      "15:04:37 [DEBUG] test episode 43: reward = -85.00, steps = 85\n",
      "15:04:37 [DEBUG] test episode 44: reward = -107.00, steps = 107\n",
      "15:04:37 [DEBUG] test episode 45: reward = -94.00, steps = 94\n",
      "15:04:37 [DEBUG] test episode 46: reward = -105.00, steps = 105\n",
      "15:04:37 [DEBUG] test episode 47: reward = -104.00, steps = 104\n",
      "15:04:37 [DEBUG] test episode 48: reward = -87.00, steps = 87\n",
      "15:04:37 [DEBUG] test episode 49: reward = -104.00, steps = 104\n",
      "15:04:37 [DEBUG] test episode 50: reward = -104.00, steps = 104\n",
      "15:04:37 [DEBUG] test episode 51: reward = -104.00, steps = 104\n",
      "15:04:37 [DEBUG] test episode 52: reward = -94.00, steps = 94\n",
      "15:04:37 [DEBUG] test episode 53: reward = -104.00, steps = 104\n",
      "15:04:37 [DEBUG] test episode 54: reward = -106.00, steps = 106\n",
      "15:04:37 [DEBUG] test episode 55: reward = -105.00, steps = 105\n",
      "15:04:37 [DEBUG] test episode 56: reward = -104.00, steps = 104\n",
      "15:04:37 [DEBUG] test episode 57: reward = -104.00, steps = 104\n",
      "15:04:37 [DEBUG] test episode 58: reward = -106.00, steps = 106\n",
      "15:04:38 [DEBUG] test episode 59: reward = -105.00, steps = 105\n",
      "15:04:38 [DEBUG] test episode 60: reward = -120.00, steps = 120\n",
      "15:04:38 [DEBUG] test episode 61: reward = -86.00, steps = 86\n",
      "15:04:38 [DEBUG] test episode 62: reward = -105.00, steps = 105\n",
      "15:04:38 [DEBUG] test episode 63: reward = -198.00, steps = 198\n",
      "15:04:38 [DEBUG] test episode 64: reward = -106.00, steps = 106\n",
      "15:04:38 [DEBUG] test episode 65: reward = -106.00, steps = 106\n",
      "15:04:38 [DEBUG] test episode 66: reward = -105.00, steps = 105\n",
      "15:04:38 [DEBUG] test episode 67: reward = -104.00, steps = 104\n",
      "15:04:38 [DEBUG] test episode 68: reward = -106.00, steps = 106\n",
      "15:04:38 [DEBUG] test episode 69: reward = -85.00, steps = 85\n",
      "15:04:38 [DEBUG] test episode 70: reward = -87.00, steps = 87\n",
      "15:04:38 [DEBUG] test episode 71: reward = -106.00, steps = 106\n",
      "15:04:38 [DEBUG] test episode 72: reward = -91.00, steps = 91\n",
      "15:04:38 [DEBUG] test episode 73: reward = -90.00, steps = 90\n",
      "15:04:38 [DEBUG] test episode 74: reward = -104.00, steps = 104\n",
      "15:04:38 [DEBUG] test episode 75: reward = -106.00, steps = 106\n",
      "15:04:38 [DEBUG] test episode 76: reward = -104.00, steps = 104\n",
      "15:04:38 [DEBUG] test episode 77: reward = -85.00, steps = 85\n",
      "15:04:38 [DEBUG] test episode 78: reward = -105.00, steps = 105\n",
      "15:04:38 [DEBUG] test episode 79: reward = -106.00, steps = 106\n",
      "15:04:38 [DEBUG] test episode 80: reward = -105.00, steps = 105\n",
      "15:04:38 [DEBUG] test episode 81: reward = -104.00, steps = 104\n",
      "15:04:38 [DEBUG] test episode 82: reward = -85.00, steps = 85\n",
      "15:04:39 [DEBUG] test episode 83: reward = -105.00, steps = 105\n",
      "15:04:39 [DEBUG] test episode 84: reward = -89.00, steps = 89\n",
      "15:04:39 [DEBUG] test episode 85: reward = -87.00, steps = 87\n",
      "15:04:39 [DEBUG] test episode 86: reward = -104.00, steps = 104\n",
      "15:04:39 [DEBUG] test episode 87: reward = -114.00, steps = 114\n",
      "15:04:39 [DEBUG] test episode 88: reward = -92.00, steps = 92\n",
      "15:04:39 [DEBUG] test episode 89: reward = -86.00, steps = 86\n",
      "15:04:39 [DEBUG] test episode 90: reward = -104.00, steps = 104\n",
      "15:04:39 [DEBUG] test episode 91: reward = -104.00, steps = 104\n",
      "15:04:39 [DEBUG] test episode 92: reward = -104.00, steps = 104\n",
      "15:04:39 [DEBUG] test episode 93: reward = -104.00, steps = 104\n",
      "15:04:39 [DEBUG] test episode 94: reward = -101.00, steps = 101\n",
      "15:04:39 [DEBUG] test episode 95: reward = -88.00, steps = 88\n",
      "15:04:39 [DEBUG] test episode 96: reward = -112.00, steps = 112\n",
      "15:04:39 [DEBUG] test episode 97: reward = -105.00, steps = 105\n",
      "15:04:39 [DEBUG] test episode 98: reward = -105.00, steps = 105\n",
      "15:04:39 [DEBUG] test episode 99: reward = -105.00, steps = 105\n",
      "15:04:39 [INFO] average episode reward = -104.77 ± 18.28\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD7CAYAAACCEpQdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO29ebwcV3nn/Xtq6e676kqyLMuS5Q2xGC+ABdiEkLAbkrANZJwNsprwSSaQZELGrycZMm/45M3yJkACvONxQkKSYRkSA4ExBhMSE8c2CDBG3rAkYyNLsva791JV5/3j1Dl1qvpUd/Vyb3fffr6fjz66Xd1dffr27ef86vc85zkkhADDMAwzXjiDHgDDMAyz/nDwZxiGGUM4+DMMw4whHPwZhmHGEA7+DMMwYwgHf4ZhmDFkzYI/ET2HiO4hovuIaB8RvcC470YiOkBEjxDRq9dqDAzDMIwdWqs6fyL6IoA/E0LcRkSvBfBuIcQPE9FlAD4G4AUAzgdwB4CnCyHCNRkIwzAM04S3hucWAGbjnzcBOBL//HoAHxdC1AA8RkQHICeCu1ud7JxzzhEXXXTRGg2VYRhmY/KNb3zjpBBiW/b4Wgb/dwG4nYj+BNJeelF8fCeAe4zHHY6PteSiiy7Cvn37+j5IhmGYjQwRPW473lPwJ6I7AJxnuesmAC8H8OtCiH8goh8H8JcAXgGALI+3ek9EdAOAGwBg9+7dvQyVYRiGMegp+AshXpF3HxF9FMA745v/G8At8c+HAVxgPHQXEksoe/6bAdwMAHv37uUmRAzDMH1iLUs9jwD4ofjnlwF4NP75swCuJ6IyEV0MYA+Ar63hOBiGYZgMa+n5/xKA9xORB6CK2L4RQjxARJ8E8CCAAMCvcKUPwzDM+rJmwV8I8W8Ars65770A3rtWr80wDMO0hlf4MgzDjCEc/BmGYcYQDv4MA+BfHjmOx08tD3oYzBjy2MllfPXRE+v+uhz8mbGnEUZ4+99+A3/1b48NeijMGPKhrxzAr3/ivnV/XQ7+zIZhpR7gV//XN3FsvtrR8x59agm1IEK1Ea3RyBgmn1PLdZxZaWC991Pn4M9sGB4+tojP3X8U33riTEfP239kHoC8AmCY9ebMSh1hJLBUC9b1dTn4MyPF/ifnccV7bsfxxWZ1v1iVX54g6kxB7X9SBv+6EfxPL9dx5Xtu73giYZhOObNcBwCcXWms6+ty8GdGisdPrWCxGuDIWVvwl1+esMPg/50nm5X/sfkqFqoBHn1qqYfRMkx7TsfBf36Vgz/D5BJEMkBXG82LwrtR/kEY4aGjCwCARpg8T00Ei+t8Kc6MF0EYYSH+u2XlzzAtUKq+FjT780vxlyiMinv3B08s60SvqfzVz+qcDLMWnDXUPit/hmmBUvV25d9IPaYIyvI5Z7qEujGhKP9/qba+X0hmvFB+PwCcXa23eGT/4eDPjBRhi+CvLp+DsHjw3//kPCZLLp6+fSal/NVEsN4VGMx4ccawetj2YZgWtLJ9uvH89z85j8t2zKLiuxnPX/68wLYPs4acNpT/Ats+DJOPDv4W5a8smqKefxgJPHh0AZfv3ATfJfb8mXXnzIoM/g6x8meYlgR9VP6PnVzCSj2Mg7+TqvPXwZ9tH2YNUcp/1+ZJ9vwZphVhgVLPsKDn//ipFQDApdumUHKdlPJXkwsrf2YtObtSx4Tv4rzZClf7MEwrkmofm/LvrNpHPa7sufBdB42guc6flT+zlpxebmDzpI9Nkz7bPgzTCqXqa4HN81d1/sWCv3qc6xB8L+P5x8p/ocqlnszacWaljs1TJWya8Fn5M0wrWin/hQ49/8AM/k2ev7xvqRase7dFZnw4s1LHlqkS5jj4M0xr8ur8a0Goa/OLVvtE8bk8h5o8fzURCAGs1JuvMhimH5xZrmNusoS5SR8r9dB6RbtWcPBnRoq8ap9FIzHbKJjwzSp/83nmal/2/Zm14vRyHVsmfWya8AGsb4sHDv7MSJFX7WNW5RT3/OW5VPAPI6Gfa14FLHLFD7MGqKZum6dK2DRZArC+C704+DMjRRHl36nn78UJXyAJ+mbwZ+XPrAWqqdvmSen5A+u70IuDPzNS5Hn+i0ZVTqeevxt7/oAZ/JMJZJErfpg1QDV12zwlPX+Agz/D5KKrfTLKf6EH5a9sHyAJ+uaVBS/0YtYCtbp3y2SJPX+GaYeu8896/rE14zrUXZ1/k/KPQNIJ4g1dmDVBdfScm/QxNyE9/7OjEvyJ6C1E9AARRUS0N3PfjUR0gIgeIaJXG8evJqLvxPd9gEh9xRimPfmev/zSbJrwu/D8Hfiu/DNUVT6NMNJqjJU/sxaopm5bpkqYqXggAuZX1q+/T6/Kfz+ANwG40zxIRJcBuB7AswFcB+BDROTGd38YwA0A9sT/rutxDMwYkVftoxK+c5N+4d4+pvIvec3Kf0tcgcEJX2YtULbP5skSHIcwW1nfhV49BX8hxENCiEcsd70ewMeFEDUhxGMADgB4ARHtADArhLhbyGWTHwXwhl7GwIwXKq5nlf9SLUDFdzDhu4WVv932kcfqgUDFdzHhux0H//955yHcdeBkR89hxo+zK3X5N1uSunhu0h8d26cFOwF837h9OD62M/45e9wKEd1ARPuIaN+JEyfWZKDMaJGv/BuYqfjwHNKbvLdDTRIOocnzr4cRfM/BdMXruNrnfXd8F5/+1pMdPYcZP04vN/TVJQDMTaxvc7e2wZ+I7iCi/ZZ/r2/1NMsx0eK4FSHEzUKIvUKIvdu2bWs3VGYMUFs0VhthqufOQjXATNnrMOEbwXMIRJR4/sr2CSKUXMJMxetokddyLcByPUytE2AYG6qpm2J2nfv7eO0eIIR4RRfnPQzgAuP2LgBH4uO7LMcZphAqsEdCKncVtBerAWYqHjzHKbyHbxgBjiOfr+v8jYRvyXMwU/Y6sn2OL9bi53MzOKY1p5fr2Gwq/8kSDp9ZXbfXXyvb57MArieiMhFdDJnY/ZoQ4iiARSK6Jq7yeSuAz6zRGJgNiOnnm9bPUmz7dKP8AcD30p6/Cv7TFa+jap+nFqoAkOoQyjA2zmaUv7R9RqTah4jeSESHAVwL4PNEdDsACCEeAPBJAA8C+AKAXxFCqG/qOwDcApkEPgjgtl7GwIwXZmA3k75a+budef6uCv4Zz78WRPBdB9NdKv+6ZZtJhjFRTd0Uc5PS9okKipdeaWv7tEIIcSuAW3Puey+A91qO7wNweS+vy4wvZmA3lf9iNcB02cNqIyys/KNIJMo/6/mHEUqug4rvduT5H4+VP3v+TCu+/f2zWKgGuGTbtD62acJHJICleoDZit/i2f2BV/gyI0WYsn1M5W9W+xRf5OVmPX+jt4+vE77Fk3AntOfPwZ/J5+Y7D2Gm4uFNz0uKHdWiwkefWsQv/+038N8+sx+ra7iXRE/Kn2HWmyBl+8gvRhgJLNdDzFQ6rfbJt310wrfi6d28iixGTzx/TvgWpdoI9aRpY9tMGRXfzb0/j2PzVTTCCI5DOH9Txfr5LdcCvdhqbtLHjKG4gzACEem/EUD+XagKsTyEEDgyX0UUCZQ8B9tnK6n7v3dyGbftP4q3/9Clqdebi5O/P/k/74WAtA7vOngKf/4Tz8Wzdsx2/P7bwcGfGSnCSMAhWe2jlL/y5KXn7xRW3UEk4Dky6OuEb5AkfJXnHwlgtRFistT+66KrfdjzL8wvfXQfvvpo/qK4ay7Zgo/fcG1H57ztO0fxjr//pr79B2+6Aj/xgt1Nj/ntf7hfNwXcNlPG129Kihvf9Yn74LsO/uw/PgeADOov+aOvYPeWSbzv+udgx6YJ62v/3b1P4Hc+vV/f/q8/8iz84g9eom/f8m+H4DkOfu5FF6Wet322DAC4ZNs0/vwnnoOj81X8xie/jTd88C585T//MM6fs79et3DwZ0aKIBSYKsvae9XcTdkys7Ht04nn7+Z4/jrhW5FfkaVq0FHwH8Zqn4eOLmB2wsfOPgeRTjixWMORs6u46oI5fUzd/plrLmx6/P+693Ecna/mnuuLDx5DJICy5+B1V52vrxCOxM/5v99wOX7n0/v1FRkgBcR//fR+fOxrT+CqC+bw0y/cjS8/dBxfeOAYgjCCF18FPnF6ReeEAPk3cXS+iqPzVVz3vq/ifdc/By99xrlN4zo2vwqHgD9681X4p28fwR9+4WG86NJzcNn5szi+UMX/3ncYb3reTpybuSK4ctccPnHDNbjqgjlUfBdPO3cGX3jnD+JLDz7V98APcPBnRowwEpgqxcE/VtcqITsd2z798fzjUs+y/Ios1gI0f82bGeaE7zs//i1csXMO/++PXzWwMdzy1UP45L7v41u/+yp9rNqI8Nzd03jz1buaHr/ve6fxzw8ft57rI3c9hg/9y0F9e7rs4bVX7IjPKYXBW67ehfd89oHU2o+vPXYaH/vaE/jZF12Em37kWfBdB2dW6vjCA8dQC5LgX2tECIzgr875tmsvxFcPnMR//6cHrcG/1ohQ9ly8+epdeNkzz8Wr/uxO/MYn78Pv/OhlePen7ocQwC+95JKm5wHACy/Zmrq9dbqM6zNXLP2CE77MSBFEESbLUt2pL2PK9umb558kfIFiWzlWG6G2EIbR9plfbWC1sbZN6qJI4CN3PYa/v/dx6/0L1QDzq43U6uzVRoiKbw9FFd9tauWhWKwG2DTh459+9cUA0t1Xa40QRPKKwHMoNRmv1OXj3vS8nfpzV1cM5mtVgxBVY0N1ZTM+47xZvPDiLVjOKQGuhxHK8fvZMlXCH7/5Sjx8bBE/dcu98FzCJ3/5WlxqVPkMClb+zEgRRgKzcVVEovyl7SMXeTkdKH9jkZfR2E3t5VtyXUyXi7d1Pr4gLZ+y5wxlwne1HhZe/dwNTy1U8eufuA//fvAUnrF9Bj/1wmYbpx5EiEQcID1Xj2siJ6Fb9p2mjXsU1UaIyZKLHXPSPjEDdS2IUPYcEMld2kwbTk0EKt8DyM9MniNdSmw+Rk0MFd+B3yK3VA8ifSUJAC995rn4zVc+HccXa/jt1zxTX00OmuEYBcMUJIhtHyD5MipV3kr5N8IIb/7wv+M3X/UMvOTpsk9UGAEOZTz/INJfat9LlP9SrX255/FFafnsnJvAyaX86pVBUQ2iwldFrThwfBG/+clvY8tUCX/4H67EubMV3PHgU/itT30b1UaEC7ZM5OY8VIVWtS6DvxACq4384D/hu6jH4zarbtT7qfiuXbU3Qn3ccyk16amJueQl51OPNTcJqgURQqd5UWHFd+Pgb/9d1oNItwhX/KeX77E+dpCw7cOMFGEkMJWxfZTVohq7BZbAs1gN8O3D83jw6IJxrgheHPRVc7dGGOnAVXINz7+A8n8qVv47N08MXW+fMBKoB1Hhq6I8Pva1J/Cjf/5veOL0Cu4+dAqvef9X8a6Pfwu/+NF92LFpAp/7tRfjBRdtzV3hrI6vxp+dDqgle/DXQTlotn6qjRBlz0FFqfaGqdojreazKl39ffiuRfk30so/e1s91s9cTZjUwubgP4wM/wgZxkAGfxmQVeBY0so/v9pHBQ9zYggyalIFCeXX+65jKP8Ctk+s/Hdtnhi6hK96/5HoPvjfe+gUbvzH7+D5F23B7e96CT73n16MbTNlfPq+I/iFF1+MW3/lRbh02zRKnmMN1nIc6eCvAmqe8rcFdoVS954rff2sX68mjqxKb9iCf2aSEUKgFkRNVxOAnJBKsVAQlt9nrZG2fYYVtn2YkSKMhC65VAFhsdqA5xAqvgPPtXv+tfix5n2REKlSPhUkVKDwXUdPNIU8/8UafJewbaaCIBKIIqG7hg4atVK0F8//0/cdwWTJxc0/sxcTJRfnzlbwmV/9ATx5ZjXVpqDsOU2b7Si08o/Hs9om+KuNTlYtSd9aI9KJYpkYTqv0iqeCfzrhq2wfddUHQD9WnaMeRhACCITQ5Z9VbftI5S+E/Hs0z6OeW+5iUdp6M/zTE8MYBJHQFRwqwbdQbWB2wgcR5bZ3UMHIvCoIQqE9fwD6Ul4FilJ8eV/xnUKbuB9fqGHbdFlbCI2CDebWAxU8u/X860GE2/YfxSsv264DMgCUPTcV+OWx/OCvlLUaj5oE8lbw2vx8hanuK77TVJmjJoas7aOu7Eop5a8SvqF+fvI66Q2Eyp6ry0Ft9l49CFEeAeU//CNkGIMwbsZW8V2t5s8sN7A57o6o2jtkL8fVF9f8smZVW8klNIJIBy6VBJ4u+4U8/+OLVWybreigMkydPav6yqe7Md114CTOrjTwY1ee3/axZc9BPbBbIsonV5/HaqN18C97LYK/oe7Lnttk0ZRzbB/1OzBtH3Ue9TdVC5rtHtP2yS4KTL1HS8J3GBn+ETKMQRBFcF1KKT1zUwxl42QVbqL8s55/8hXwvdjzD9PKUPX3acfxhRq2z5R1YBimpG+1R+X/T98+gtmKpyulWqGCri0wquCqFL/2/HMSvuq4Pfibto+jzw3Izzvx/NO2T8Nm+8TnUUG/Zkn0qmMV39HB3ZbbqXHwZ5j+o5R/2TOUv7Ephht/obPWj/pSmwE53/NPK8PNkz6+/thp7H9yvuXYji9Wce5sGaVYRQ5T0lcFsG6qfaqNELc/cAyvuXxHoaDW6spHTQgr2vaRt3tJ+AJSja9mlH8lp9pHjct3mhO+WZUPJMJB/Q2pUk/A/hnXg6TSaJgZ/hEyjIFS62VD+Z9ZqeuNsHOVf6M7zx8AbvqRZ0FA4I0fugt/fddj1nHVghBnVho4d6aSWjMwLBT1/IMwwr2HTqWOfeXh41iuh3jdc9pbPkDin9t8f/U5VLPKvwvPf9UI/hOZlcC1IEm6Zuv81eI+MxmvJook0DcrfzUBqVJPIGkEaFLnUk+G6S9RJCCEDPAVz0Ut3sT9zHIDc1PK85d/0lmFqyYK0/NWVxEKVb5XC9LK/+oLt+AL73wJrrlkK37vcw/izHLzVnuqJfH22XJLS2BQVC2Tn42vPHIC//Hme/D4qWV97N8PnsJ02cMLL95S6LWU8rcG/2zCV9s+9lDUvtonUf5Ni7wM5V/P2D5+JiHbSvmr353N87cl9bMrfIeV4R8hw8SogO46hLIvK0qW6yHqYVRY+WcVoOta6vwtK0A3T5XwY1edDyHsNf+qm6dU/rHtMUTBv6jyV60yzqwkK5qXagE2T/m6wqUdSvlbbZ9Mnb/6XyV2s2RLMBVhJFAP055/U6lnHNBLrpOa9OtB1FSembWXzHOpVb/VIITrEHzXaWoEmH2PrPwZpo+ERvCvxNUdSoVrz99Rnn/6S6lUaLrOH3Aztk8jEKlFXibKx7UF9flVGSw3TfpJYLBYAoNC2SztPH81Oa4YE9xyLdAtNYpQcvNX5epFXgUTviq4Z20f038HpHLPK/X0XEp9FmqLThO1UEwnfM1zGeWf5tUEYP+MZV8hrvNnmL6hArpa0FVtRDizEgf/WPmry/Hmap/m4Gc2dgNktY/p+WeDfyu1pytBPFdvDDNMyl8FsHbKX4152dg+cKUuG6gVRU+SQbNaV79/XepZb+35Z+0YRVX/vmPlbxQAyNW52RW+ZnuHZttHjdum/E3bR5+zxWfMyp9h+oyp/Muei1oQ6i34tmQ9/zAb/JXtY3j+Ybq9Qynb2yfzBS7lBDV5/ti+8B2j1HN4gr9e4dumzl/9flTbYwBYrhfbyEZRyiRPFebvbTWTRM2r85/IDf5p5S/FQFLRFQmkg7/xvhths+2jzmX3/A3lb5SPqnOZCCE44csw/UapRlP5n4296Wydf1PC11LqGGQWeSmFqIJU1hrwW5QwqkBX9pyhXuTVttonvn+5Zij/Wn+Ufyr4G+0dSp7T1LFT4bsEh5o9/+bgbwRuNRFriyZt+9Qtto86h63axyz1VOfMuwpUwoFLPRmmjyTK34m/qInyV8Hf1QnfHM/f+LJGornUM9vbx6S18k8UbKsa8EFRNOFbtyj/lUagexwVIVH+dp9enjNR2HmWDyC7rdo2dKkaC67U/01tGAzlb17xtLZ9Wiv/pHzU/hnXcoTDMDL8I2SYGFP5K3/2zEodDkFv8JKn/G2N3YKoeZFXqp9/xhrQQc3q+SdqczhLPTtM+NZ7Uf7xCt/AHhiBJAHdaiMXxURmARdgqHul/D0XYSQX6NUy+QD1uSoaYQTfa77SKOcofzXRyDxCcjUh32P696lep5yzM9kwMfwjZJiYMDSqfWI1eGaljrnJklb8utqnyfNv7mopPf/kK1DyKN3eIev56wqPVraPa5R6dlbtc/sDx/AX//xoR8/J44sPHMOffum7+nbR9g4NnfBNe/6dKP+8RV41i+ffagtHRbZjJ2DYPl5i+6jzZS0h2d4hbfuYO3Qlr9NO+Se9hHJtn3FR/kT0FiJ6gIgiItprHH8lEX2DiL4T//8y476r4+MHiOgDRGQ3+xgmQyiSnixKpZ1ermMubuqm7gOag5zN8w6F3fPPLvJStCr1VHvG+i61nCRa8fn7j+Lv7nmio+fk8Ymvfx8fvft7+vZqQeXf0KWeyWRRbUQdKf+8RV62hK+5SjePcqZjJ5Dus2P+b27AUsmxfWylnkDcjVRV9gQyF1FynXSpp9EpVJ3L9h7HIeG7H8CbANyZOX4SwI8JIa4A8DYAf2vc92EANwDYE/+7rscxMGOC8vHd2PYBgGPzVb3AS95nX+Gre/tkGrvZPX978G+X8FV7xipLodNSz2ojbApy3XLwxBKWqoHurFlU+QcZ5a+8/45sn1zlL8dQcp1UnX9ejb9iwne1TaTIqnu9GUsjMtYAqDp/+bmq30UQCqvto/JI6jwVT7YRqbUo9cxL+G744C+EeEgI8Yjl+LeEEEfimw8AqBBRmYh2AJgVQtwt5CfxUQBv6GUMzPiQrvaRX8Jj81W9wAsA/LZdPTMtnbOef2z7uA41VaC0S/iWvURpAp17/tXMzlHdUg8ifP/MKoJIGDtnJe/f1mpZoca8anjyADoq9Sy7ds9f3d406aeslHaefyWzgAtIPH+z2kedL6v8S5kuq40walvnXwtkS+hsFVFzqac9t8SLvCT/AcC3hBA1ADsBHDbuOxwfs0JENxDRPiLad+LEiTUeJjPsBKFZ7SP/dJ9arOle/vI+5fnb/Wb1ZRVCNG0Kruv8g6gp2QsYwd/axtcoA2wxSbSi1ghRy+mD3wlPnF7Wk5zah8CcVFqJ/4Yq9YyDvvpf7ZtchHKmPbJCfQabJvyU7dM++DtNnn+2G2iyHiCZQNXnoSpzlPVTD0WO5+82WTwVP9mYpmbsC5xf6hlf3WwE5U9EdxDRfsu/1xd47rMB/CGAt6tDlofl/ikKIW4WQuwVQuzdtq19H3FmYxOmqn1cfcxU/l5OS+fE9pBfVnV3dg9fIeQX3+YJt1T+jUgHvSQwdBbEq4HcOrDXlcEHjidN2VQfIjP4t1ropRd5xc9brinbp5P2Dq3r/Ocm/NSVRd7m7YoJ39WPVyS2T8bzD8Kmq4JsK4YgjFJ9mxTmKuFaIJO7FS+98KvpnCNc6tn2ExVCvKKbExPRLgC3AnirEOJgfPgwgF3Gw3YBOJJ9LsPYMBu7mRUiNs8/z/ZRVw+BkT9QKB93uRZYlZsOajkbeGRtn26UPxDXk/dgGxw6uaR/Vo3azODfyvdXE5ZS/Krks5PePo5D8F3KrfbZNOGn2ihU2rzXbN8eoKDt42Vsn0hd/eXYPn46uVvO9BWqBskxP2eCH6eErxUimgPweQA3CiHuUseFEEcBLBLRNXGVz1sBfGYtxsBsPLKN3RSbjeCfX+efrnYxryIU6gu9XA+swaHVyl3T9nEduSq1Y89f7xjVm+9/6ISh/GPbZzWl/Nt7/irRqxK/7ZKyWUpuemctILGBNk36qIcRgjCStk9OO2eFqcgVZm999Rh1PHtVkF2Q1Whl+2TKOqXyjxBFAvUgSm0KD+SvZdjwK3yJ6I1EdBjAtQA+T0S3x3f9KoCnAfgdIrov/ndufN87ANwC4ACAgwBu62UMzPhgqnVzEY1p++St8K1r5Z9e7JX1/AGpdm3B33HkBvGtqn30uTyni+Afpf7vlkMnljBbkUpdbTyvPHIgWS9hI7vIa7ULzx+Qal3534q6ofyBpCa/7SKvktO0yKvWCHV1FZAt9Wxe4Qsktk8j1/ZxdM6lFqt8lQcwV3ADcuWx51BuqecoBP/i13IWhBC3Qlo72eO/D+D3c56zD8DlvbwuM56Yat11koChmrqp+wBbqWc66EeW4K+Vfy2wJnwBGdRzPX/jaiS7gUgRtOXQQ7mnEAIHTyzjqgvm8NVHT2rlX2uEKMVjKqT8M55/J7YPkK6ZV9S05y8n69V6WCzh69naO6TXB5i2TxKo06tx29s+rs65VBsh5iZ8EAGnlpuvJuR5myf4sbd9GGYtCArYPm5OqWe2vUHQyvaphXof3iwlzx7UZWmgofwtgaEdttWlnXJ6uY751Qau3LUJQJLwXW2EWr239Pzj+1YaIaJI6CuATur8AfvvSSd84+qs+dUGhEDbhK+yY8wqKHPBFZBUGFWDSC+4UzZdNjlr28kLSNR6LYj0BvDluPY/q/zleanZ8x+XOn+GWU+UXeHFe/gq0p5/XNbXpqWz2SROoRO+9UBbQFnygnrW9sn2k2mHshrMsXbDoZPS779y1xwAGfwbsdqfjq2gsFWdf/zaQsgrEOX5d9LeAchT/nIiUcFfNeVrb/u4iDJVUGbNPWAo/3ooE7OGJaQCvfqbqOe0dC6nksbSVlL5hmz5KNB6ghuFap/hHyHDxNiUv9nUDchv75DdzCU5V/KYdp4/IAOJfW/ajO3jNavCVtRDWeYJ9Kb8Dx6XlT7POm8WJdfBQrWhz6esm5aev5ErWa6FWKmFcKhzDzsvMBIBM/EkpDbiaRf81Ws3bdNo/L6ThG/YZAmpvwk1niCvpbNS/o1Id/BU/X6y1UWA2vnNPsGV27ynYYCDPzMyaM/fTUo9zaZuQGLjNDIbtTfCdNCPbMo/5fnbvxrlXM8/TKvCDj1/285R3XDo5DJKnoOdmycwU/GwVA30+VTQbVXnb05Yq/Uw3sXLQ6ctuNRmOya1eGPzCV8Ff1mG2q63j7rfrILK2j66738QNk0MZq+lMJIbvdg+X/06Qag7eDzVv3EAACAASURBVCrLKdtCWr5mC89/BJR/TwlfhllPUtU+8ZfbbOqm7gPSyt/sKRMW8PxrLbbhy034GjXg6lydNHZLB7bulf+hE0u4eOsUXIcwXfGwVAsS5R9bN63r/A3lXw+wUg869vuBePKzlEGWPUeXjSrbp13wV1cGq5nfkamuk77/kWViSPo9qfdntX2MKwyVwJcTipHw9TKef+Z3qd5zXsHAMDH80xPDxKRW+MZfbnOBl7yv2fNX3vNk2dW9bULbIi+3OWBkyU/4pm2fTks9s5ZGtxw6sYxLtk0BAKbLUvmroDldVsq/damnCvYr9QDL9bBjvx+QCVjbIq+S5+pgfkZ5/gUSvkDmdxRETZOGUum1IN/20e26Wyj/lXqIehhp5R9GQlc9ldvZPvEWjqPQrJiDPzMymJ6/UmlmjT8AuBbPXwUh5XkHkYCKy6k6f8/+s0l+wjdsTvh2EvwNi6TaZcJXCIEnTq/gwq1J8F80lP90EeUfRZiLcyjS8++f8q8r5e+nlX+R3j5AelKsNUJMZPYBmDCUvxmkTdsnb5c2+TryOQuriR2lXvustqhar+Uw+/8MO6MxSoZB4tN7jlRWZc9JNXWT9zXX+SvbRyvfUNjbOxRV/pmgpnIK5awlEBRP+GYDWzfUAlnVMzsh36fy/JOFWsVsH5VAl8o/6LjGH0jvipWML66giVf0ni6Y8LXZPrZ9AFR7hmom/2L2e8pr1w0kts98HPzLnqNfIzmWUf6WUk8O/gzTZ7Krcq+7/Dz84J50wz/bCl+t/OM69yCKWrZ3yP5sYivhTCo8TFXodqT809sGdhf8dU1+HLCk8m/oK4mits8mHfxDuc1iH5V/yVD+ie3TOgyZJZiKbFIXUG0gwiZLyKzzV2Oyef7qOWdN5e+lg382yWyraBqFZC/ACV9mhMgG7Pdf/9ymx9iUfzbhKZV//gpfIH+RTsmzeNmZPjNA0h66KCnl36Xtk2y8opS/n1L+hWyfMNLBf7keYrkeYufmzoO/9Pybq31M20dV+7RrYpfYPumKqOz2j6r1c60RojJT1sfNLqvqc7d7/s3KX/19zK82Vyb5rqMX0SnUBDcKcPBnRgYdsFtUUhDJTVjaef629g5mQMhTb7aEr7l/r6LTRV79SPhq5R9f4WSrfaYLlnpq5V8LYs+/C9snZ5FX2XPhuXJ7xM4Tvhnln5PwzavzN/dntts+zZ6/+vs4u9JcmZRX6jkKG7kAbPswI4Su0GlTSeE6lPb8jWofQAY/q/L3zKsA+2uULV94bftkEr7dKv9u6/yzrRimyx4aodCqtYjnH4QRZiqyp41S/lPd2D6eg5rNElFdOH1HN50r6vmr31HSeM0S/IMwt9Szve2jkrt1fVt9ptoK8tK2TzavU4v3/h0FRmOUDAN7J04bnkOpnbxsCd/QSB4ruk342vq+yEqQ7hK+3Sv/rO0j/z+xWAMATOvJr3U//5LnYNJ3pfKvB5jsptTTc1HP7EpmtsAw1X7RRV665XWmcVvyOLk3cLbU0y9o+5Qz/n7Zc5M8wEoDnkO6PbQ6r20PXw7+DNNnkt4+nSl/paSnjISnrb1DEc/fmvC1eP6dl3omSdluSz1Xas3KHwBOLsngX6S9QyOSW1hOlj3MrzbQCEVXyl/9LszfQd1YCKfUfMnw1fNQQV7tQ2xbcKVuJ4u80pVXQBHbJ+35qzp/dSw7SdlWcY9Swnc0Rskw6Ez521b4qiAWRpG1vUOpa+VvqfbpMOGryjvlLlddKv9GerN1FfyV8te2T05jN7kATr73qZKrJ42JLj1/IJ28rhmBUQXSdpYPkO7bI/9vvtICoDtwVi1rLgBpaalFWTZbz3EIJc/JKP9kQsiWcPqu09RAcJQSvqMxSoYB9Ibr7VZPuo6TqfNPK3/z8j9d6mlP/pqsVcJX97qf7CH46/125ThmKjJxe2Kphorv6CCY5/mbrQ8mSh5O6CuG7jx/IL3TlZkMVbZP1rqxoYJyssVic299dXuhGsg20WbC11ErfIX+7Lycz7fsOZhfDfT5zFLP7GQjm/fZW1iMAqMxSoaBVP7tVD8QK/9Ue4fmRU629g5yYpE/527mEi/siSxXFim12XF7B9k9c7rs9VDqmbZ9TM/frFzJ8/y1JeJI5a+uGLrz/G3KP0mGTnSg/AGZaK3W05vd2Kp9bDtpEVHcez/SSj1vcq/4LuZX6/E5Es8/jETqyg6wW3us/BlmDQijqK3fD1g8/8wiJ9XfXj1WIYOE/Er4OV9g9cU2u4Zqz79pMxeRSni2QpUnVny36xW+2YSv6flP+K7+3WW3uFQEuvWB9PxPLskg2F/l78RjTG+83o6Jktu0zWWT8jdbPGdVuutI20dNcDntOyp+kqg32ztkzw/YW320ago4bIzGKBkGHSh/l9IrfFWpZylRcWFO/kApwlzbx7KJu8320ZNEwYoflaRUC5W6YaUewostEiCp62+EIq38c8bUMCyRqZKrf0fd1fkn7ZEVZmDUnn/BiUWVcQItEr5moM4Ef7nfrmH7WDZwN8cNyMm8nJpQmpU/t3dgmHUgjEQh5e9l6/yDEL5LqYBsa+8AJHZPqxW+QDb42+r80xuItEPvHGUEuU6RvfeTYDVt2DVm8M/1/CND+RsBv9vGbkDyewrjCivt+Xds+7h6pbIO/plxpctH05+fytW0t33SSt/8TG1XE6aQAHiRF8OsCVL5t/+T9Zx0FYZqt6zUXhjZ2zsASWVIq2ofIB3UaxafWdeWF/TvVT+asuf0VOdvBu2y5+hJqOI7ie2TY0UllTBOKuCrnkidoCww9bvJbmyuAnXh4F9ytX3XqtTT9jOg/iba2z4qcBPFG8QYV1JZRW+uHFbwIi+GWQPCsJjyb1rhG+/KpL+sqVLPDoO/zfbRnr+9mVgRUsq/B9tn0gjURKQrfiYKKH/V9sFzndR5erF96pngX84kfNtt3q6oGJNinudv5lyakrPxtpqt6vzNc1Y8V1eVVTJWlaJk+Yy5zp9h1oBuPf9qQyn/OPjlNHYDEmWaW+3jNX/hbbaP7QqhFbUg0gnfXnr7ZC0aZf3IhK+qd8/z/GPbx6FUG+duWjqr969+N3o3tUwgzSr0PMxEeFLq2VztY/sZSFbj1vV7zAn+XnMJqh5r0zmV8o93hwsjRCLfMhw2RmOUDAMgEsLakyVLs/KXSTgd/FItnbNJvNaev7nVo3l+h9L5A7OTZBFktY+DStw1tGiVkMlKPcCknw7UKvhXfNe60Y2JqYrNSaSbls7lTG4ka41p26dNO2c9Bt/V/fwLBf/MpOI7Tlzq2cb28ZXF0zyRNCV8M0JATfSc8GWYPtNRnX+qsZv0Yc1NPbTyJ7vtk3fpng1qQJJTMBef+RZ7qBW1uNRTWUfd1PpnbR8gqfipGKWe+XX+8YTokl4TYSbKOyFR/ung33Wdv1EFpbz/5lJPJ/V4k8K2j1X5N08I5jn0BNdIv8dhp6dREtFbiOgBIoqIaK/l/t1EtERE/9k4djURfYeIDhDRB2gUNrtkhoJu6/xVB0jPKHXUnr/bZcI3s/lK8wKg5mRgK6rx9n9KZWbbIRfBZvvMaOXvwKF2df5x8DKUfzd+P2As8mqo4K+ssXSJZ/Hg7xqef16pZ3HbJ+/vqGyxePJsHyUQ1N+aUv5jEfwB7AfwJgB35tz/ZwBuyxz7MIAbAOyJ/13X4xiYMSEIhQ5grfAcJ6X85a5Ppu1jb+8AJF/ojqp9LPu2dur5VwO1yMvRtzvF1ntfKf/0Ii95nxACf/SFh3H4zAoAU/k7+jzdLPACDOWvLJFeE74p20cmVZ3MZ2cG56bJ2LB9fDe/RYgan/l5lnWeornOHzBsnyCZPEeBnkYphHhICPGI7T4iegOAQwAeMI7tADArhLhbSFPzowDe0MsYmPEhjIp5/p6bo/yV7RNG1vYOQOIF59b5Wyo81CYl1scVLfWMtyVU5+km6bvSsCh/w/Zx4vYV6r0/tVDDh/7lIL780HE5Vl3tQzrod9PaATAWeWXaMGeDfyfKv2Z09cwGd/kYM2Db+vBI2ydvYlevA6Qrt/SxJtsnXsuRY20NO2sySiKaAvDbAH4vc9dOAIeN24fjYwzTluJ1/pkVvjrh27vnb/Pybc28kmRgJyt8HeuWhUWRtk824RuXesbB3FwApyYwNdGYC6BU0O9W+WdbOmvlr1o6d9jeoeLLRVphJJr69SePyV/kldT5izbB36b8CyZ8LSu9h5m20zoR3QHgPMtdNwkhPpPztN8D8GdCiKXM5ZVNtuV+O4joBkiLCLt37243VGaDU3SFr+tQZpGXrKFXnRyDUNb5E6HJOkh6+7Qu9cxW+9iafgFAPSym4FVAy7YvLkoQ71LVSvkDSG1xWdfBX/5vdvVMevB0p/zV5Jl4/soSSQf9osrf3M3Ltn8vkOQAiJonb9mETSn//L+hJNA3TyT5df7y92kr+R1m2n6yQohXdHHeFwJ4MxH9EYA5ABERVQH8A4BdxuN2ATjS4rVvBnAzAOzdu7fz2jdmQxFEUcFqn0xL57hvjptR/raJpJ3nb6/2abZ9Ekug/Z+tEEImfP2ki2Sn1T5JL397nb8KYObvRiv/OGjp4O8kVyDdtHYA5KTqu5Sr/M+dKYNI/l+ESir4h9b1AWXLAi1FySO9wreI8rcnfAt6/hsl+HeDEOIH1c9E9B4AS0KIv4hvLxLRNQDuBfBWAH++FmNgNh5h1PqSXdG8gXuYanWg+rHYJhL1mF4TvuWMJdAK0w9PbJ/OlH+yi5e9zn/CovzV/rM1rfwT20ct7OrW8wekis5W+6jJ9YItk7jzt16KXZsnCp0rSYRH1s3b5es5ILLvEeDFCd92to+axNM9fZJJJXXOTP+msar2IaI3EtFhANcC+DwR3V7gae8AcAuAAwAOorkaiGGsdFLnH0TZ4Nys/G0bwavAkHfpntfV07bLE1Aw+Bs7U2X3qy1K0s45v84fUGWw6WCllH9g2D5qvUC3nj8gf4fK9soqf0BOAEUrvdX4V+vS9rHZRUSEitGD30R14KyHUcuigUT5G8HfYgUBzUl9214Cw0xPyl8IcSuAW9s85j2Z2/sAXN7L6zLjSSeef5ht7OY7ekm/2sDdqvx1e4fWyr9dtU8ni7ySzUmclMLthOxGLorE85fnlcofqfegJpqkq6eDkisT5N2s7lWUPMfi+XcXGM29dFcboX5fzY9zcoI/6VLPVmOwtZ3Qq35zbR/l+Y+W8l8T24dh1oIgLFjtY5R6RpFUexUvKXVU7R1sW/mVXAcO5e8TnFvt08YPboW5aKnbUs8k+Ke/0s+9YDN+9kUX4fkXbQGQroRSY9O2j7G/LRHh3dc9A9dcsrWjcZiUjS0vE+Xf3WTynAvmMF328IdfeBir9RDbcnIFqjNqFrXIq73t0xzoK9oKyuvtM4Z1/gyznhTv558s8qpl7AY/TngGkX3B2HTZ060NbGRru4GcRV662qd9wtfckFy3Qu7W9sm0d5gouXjP656tu3u6rUo9ja6eAHDDSy7FlbvmOhqHSVr5pz3/Ttk+W8Hv/thl+Npjp/HIU4u5JaJmiwwTtdl6u2ofm/Jvl/DNTnCjovxHY5QMg7jap8PGbtnyO1kGKks9bRPJz/3ARfjbX3hh7rmJZK+bWsb2afKDO0r4mraPUv79sX2ymH2PVCVSUu2TbObSD8qemwqMqkd+t7zl6l145WXbAaT7+KRf07HepyqPGmGUu3k7kFftk1Pq6SU2IpCsZt4wdf4MMyx0spNXk/KPv5DKEspLHm+dLmPrdOvyw7LrFEj4xpZAEc+/kYyx2zp/FfzbtV+2K/+0DZTX7rhTSp5jtHSWXnsvrbyICH/wpitw/+GzOH/OXiV04dZJbJrwm46btk+rtQW62sdQ+bu3TmGy5OKcqfTfBZd6Msw6UbTax42Te4Cx0Yqn6twpTvi2rvpoRclzMgnf5q373Di/UKS3T9KiWJajOtRFnX9s+7RL0JrJcP07CpIVvo5l4Vu3lD0n1fqgH1Uw50yX8a+/9dLcc33wJ59nnWB810Ek5O96NidZDNjLOl+y5xzc97uvagrqWc9/1BZ5jcYoGQbItWqymMo/qaRRyt9pWepZBN9Q/kHcciD7hSeieFVpJ8FfLk7qZkOXoraP6zh6G0eb8i+yjqIopXhvAkBt3t4fO0T9nmx4rmMVCGqiX62HLW2fXZsn8bJnnourL9qsjymrL0uu5z8iCV9W/szIUFj5x0ldIUSO8o8QiWLnslHKKFqguQwQkPZQo8AK32x/+m42cV+pBXKBU5sAm/L84ysAXerZphKmU8qZhO8gFbEKyCv1sG2p51/97PMLnTPZp1nlUGTL8X5dOa01ozFFMQyQW5ufRV0dRMK4FFftDVwZ/GTZaA/BP8wEf0vQ9TP2UB4q+JqrS1slfI8vVPV+BIqVeoiJuHNnK1Kef5BW/kEPVpiNUibha5sg1wv1vlbqQd8S2q4jLTrT8x8Vywfg4M+MELIfT7H2DvLxUXPC13HQaNHeoQgl11T++T6v71KhRV61RtqaamX7HF+o4sV/+BV88cGnUsdlO+f2F/L2Ov+kt0//lX864Tso1PtabYR9fY8qkQwoa2t0QurojJQZezpV/qr9L2A2NpPBL6+xWxGk8leWSb7tYwaGVtQytk8r5f/wsUXUwwhHzq6mjsuNXNp76mbH0+bGbgJ+Hy2L7CKvbhd49QOzA2crz7+b85rvkYM/w6wBQcFtHHXr5kikyigBGfwaoejN83cd1HUJY9qyaXpcF7ZPxXf1ebMcPLEEAFiqBanjti0cbbgWz78Rt7sIwki3t+gH2UVe5QEqf9POKvXR2jKtvXrIwZ9h1oSOlX8ommwZ33W051/EQrKRSvhmEsrZxxXz/CM4xgKoiu/k7uF76MQygB6Df6baB5DBuRF2fzVko+y5qW0cB+n5m1ZPf20fSiV8R2WBF8DBnxkhilo1aoJoRFESnI3GZo24PLPbtUyFE75xJ8l2qBbFqnyxVbXPoZN5yr95/14bZrWPuQCt2mjf675T1CQphBgCzz/5u+mn7eO7jt7+ctDvsVNGZ6TMWBNFAkKg8DaOgLxSyFoqvqr2iaLulb8t4Wv1/IslfKuZ9hAVLz/he/B4rPyr3Sp/p8nzB+QEFBTcL6Eo5laOw6T8+2n7lIwJvhaEbPswTL9R5YlFShF1tU8osBgHSdUCWCU8Q5HfubMdRW2f4ou8olQ/mrJvT/gu1wIcW6gC6N72sdX5yzGEcd+b/iZ8AamIB62K1872cVL9/Dn4M0yfUQGrkOdv7Ng1v9rAZMlN9uZ1nbilc7HksQ0/pfbybZ/inn8x5f/YyWX9szX4F9h1y3WTzVzSyj+2ffrU1wdIb3k5aD/cnNT6mdT2PUolfLnOn2H6jG43XHCFr3yODP5moy+1yCkIRdcrMdNtC/Lr/EsdlHqairHiO9bgryp9dm+ZtNg+ASYLlFKmPH8z+McJ37yN67tBvaeF1QaW68FAVbF51dHPpLafKfXk4M8wfaYj5e+klb8Z/L3Y845E95UtsmFZsngJaFHnX6S9Q1b5+651J6+DJ5bhEPDs82exXE+CfxQJrDYKKn9q7uqpxhCE3edBbCil/9O33IvFaoC9Rr+c9Sbl+fdT+Y/wIi/u7cOMBNrz7yD4B1GE+dUGZlPBX9oeRfsE2UhV+zTy6/x9r5jnX2tEqY1Cyr6rq2TMBmaHTizhgi2T2DJVSin/ahBCiPZN3QA5earWEHVjYqoF7Xe56hTT8//Izz0fL33GuX07d6ekbJ8+l3qq/Ex9xKp9OPgzI0Gi/AtU+2Q8/12bJ1P3BX1t79Aq4Vu82mfLVEnfVhNBLYhSVwQHTyzjknOmMF32sGh4/kU7egLpLS4b8aKkehChFid8+9X3BgCuuWQrfu3le/DT1+zGuTOVvp23G9bS9lmsBhBC4ORSDVumWu8FMUyMzjTFjDVJ8G//WNPzX2iyfWS1Ty+N3VRveNk+osUir4Kef7URprcNtGzoEkUCj51cwqXbpjFd9nQSFQBWavb9e224Gc9f9bavNuKWF31UrpunSviNVz594IEfSNf299v2qQcRFlYDrNRDnD83+PdaFA7+zEjQkfI3Sj2bPP94hW8vnn/JqGKpBSFch+ybwXewwjdt+zj6uOLI/CqqjQiXbJvGdBywl2P1v9KI9+8tVOrppJS/2tu3ugbKf5jw18j2URP8k3GvpbwdxoYRDv7MSNCJ568UfS0IsVwPm5R/I1Sef/ftHYA4+Fs2b1eYm760otoIUzkDm/JXbR0u3TalN5hX5Z7LteK2T7a3z4xW/mHfSz2HidKa1fnLXlFH52Xw37GJlT/D9BXVhriTap/Ty3UAwKaJxA5RwU96/t2NRQX/Whi23J6w6CIv6e2bpZ5x8DdaPHz3qUUAwCXbpjGTCf6r9c5sH13nH0RJ8A8i2e9ogyp/88qsn+/Rdx0ERpfVnaz8Gaa/dKP8dfCfTJS/r7Zx7KGsUXWnVLZP3uKl2QkPjVBoeyaP5lLPtO1zYrGGD//LQVyxcxPOmS41KX+1f2/xah/5cyOMMF1OlH+9z719hgk/1dWzn4u8ZHvvI/NV+C7hnGlO+DJMX1H9aIopf/lnnSj/zCKvMELUY3sHQLZCXqmHKdVuolSgsgRsqKZnZq97Fdxv+85RhJHAjf94PxZrAf70x68CEWnPX5V7dlTt46RX+FZ8F75LqMXKf+N6/mtj+yjP/8jZVZy3qTIyWzgCPQZ/InoLET1ARBER7c3cdyUR3R3f/x0iqsTHr45vHyCiD1DeTswMY6B86k56+5yKg/9sxUz4xit8e2zvAEjlf+D4Ei46Z8r6uB2bZPB/8mw191xK3U8YwX/vhZvxuqvOx/+48xBe+af/ijseOo53v/oZ2LN9BgCabB81yZnlonm4DiESsnpI1fWrdhJyG8eNqQf9NbN9ZA7p6Nmq/rxHhV4/6f0A3gTgTvMgEXkA/g7ALwshng3ghwE04rs/DOAGAHvif9f1OAZmDAg6qPZR6vX0UrPy95ykzr+X9g4AsFBt4NHjS7hi5ybr41TZ39Gz+cpfJXUnfDM4OXj/9c/Bn7zlKhxbqOIHnrYVP/8DF+v7s7bPqeUaPIdSk1weevWzENrmKftu3Nunv4u8hgm13y7QZ9vHqPYZJb8f6HGRlxDiIQCwiPdXAbhfCPHt+HGn4sftADArhLg7vv1RAG8AcFsv42A2PmEvnn+mvUMYCYQFz2VDBf9vf/8swkjg2efbg//22QqIkNpy8auPnoBLhBc97RwAck9ZACnPH5DfqTdfvQuvfNZ2VEpOaqLKlnqeWqpjy1Sp0GSmJs8wEnKRl0uo+I4+Vz+3cRw2vLj6qv97+Ao8tVAdqUofYO08/6cDEER0OxF9k4jeHR/fCeCw8bjD8TErRHQDEe0jon0nTpxYo6Eyo0DQUbWP/LM+tVwDgKb2DopeVvgCwLeeOAsAuGKXPfj7roPtMxUcmU9sn9//3EN4/5cf1bdV8J/I8es3TfpNCeWpuKpHtas+uVTH1oKJxqT1hUAjDoQV38ViVV6Y97Pj5bChPrd+5jXUuYJIjFSNP1BA+RPRHQDOs9x1kxDiMy3O+2IAzwewAuDLRPQNAAuWx+Z2vhJC3AzgZgDYu3dv+w5ZzIalI+XvJsq/7DkpVW162m6X6Sal/L/x+BlsmSrh/BaK7/y5ilb+YSTw2KllPNOf0ferMs2s8m+F6xAmS27K9jlnur3fD0BfHYRh7Pl7Diq+o8/Vz9YHw4YK1P1W/opRWt0LFAj+QohXdHHewwD+VQhxEgCI6P8AeB5kHmCX8bhdAI50cX5mzEg8/+J1/mdXG9iWUcQp5d+lAlQK8thCFT+45xyb7anZMTeBB49IzfPkmVXUg0gHfMD0/DvrdT9d9lK2z4VbJts8Q2I2vdOev+fqq4iN6vkDycS/dsF/tJT/Wn3StwO4kogm4+TvDwF4UAhxFMAiEV0TV/m8FUDe1QPDaMJQKf/2f7JqghAi7fcD6UqPXj1/ALnJXsXOuQkcObsKIQQOxvvvrhjBv53tk4fZ3O3UUq2w7ZOsfpY2mvL8xyH4r4ntY/wtjFW1DxG9kYgOA7gWwOeJ6HYAEEKcAfCnAL4O4D4A3xRCfD5+2jsA3ALgAICD4GQvU4BulD9gCf4pz7+39g4AcHmb4L9jUwW1IMLp5ToOHpfB32zbYCv1LMJ0xcNSNcBqXbaw2FrQ9lHvX006qtRT2z4btM4fSN5bP8tZ1X7A02VPN8kbFXqt9rkVwK059/0dpM2TPb4PwOW9vC4zfnRT5w+kk73y+abn391YOlH+ygo4craKQ/E2jDbl34nnDyS2z8klmdQuurJU/W6U9aQSvir4b9RFXkByVdPvUk9ATvKjtmRp417jMRsKVe3jFPiCmdZFVvm7Kc+/S+UfP2/ThI9dm1tf6p8fWwFH5ldxKN6GcbURQgg5mVV1wrezsUyVPSzVAr2QrWjCV02e6urD9xyUfUdPrhvZ9vHXpNpHnnPU/H6Agz8zIkSi8zp/oDn4+/3w/OMv/OU7Z9uqPVUBcuTsKg6eSDZgV577apcJ3xkV/GPlv7XgJiJq8lSvKz1/oxpqg3b1BORnT9R9ia/9nCr4j1alD8DBnxkROuntY5ZwZm0f0+fvNgiUfQdE7f1+QLZcKHsOvvvUEk4s1nDerAwSyvrpOuFbUcFfKv/inn+6aZzy/BWlPm7gPmz4rgPfcfpqz6jf1/kjluwFeBtHZkToxPN34qX8kaXax1zB2m2df8V38cGffB6ef9GWto8lIpw/N4G7DpwEIK8Wji1UddDXdf45nUHzmCrLhO+JDpW/a0n4mpvPb2Tl7znU95yG9vzZ9mGYtaGTah8gCWKtPP9eKltee8UObJspFnB3bKrgidMrAKBbQazGbZjlRi5Ox32GpssegkhuIjJVcgtfOSirPl2gWQAAFEdJREFUq1pPV/vo+zdwwrfkOX1fwawS7U/fPt3X864HrPyZkSBZ4Vvsy+s6BIQ2z79326dTVDLQdQjPPE+u7l2tS9ul2gg7tnwA6D78j59aKVzjDyQL27Tn71Eq2dzPSphhw3edvl/ZPGvHLO658eU4b8T6+gCs/JkRoXPlLx+Xrb12+2D7dIpq/7B7y6TOQWjbJ7N5e1FU8P/eqeXClT5ATp1/TvuLjYbnkK7L7yejGPgBDv7MiKC2cSxaoaMUrrmLF5C2NdZb+V+6bUoHWrX71moj6k75x5Pak2dWO1P+1jp/0/PfuLaPvwa2zyjDtg8zEnSr/JtX+K7Nph6tUMnAS7ZN6922qkbCt9MFXkCi/CNRvMYfSK521P7AWeVf2sDB8WeuuTDVXnvc4eDPjARJb5/eEr5p5b8+gU41XXv69hldz69KPauNMLWRS1FU8AeKV/oAxiKvuqrzd1Itozey8r/mkq2DHsJQwcGfGQk6Vf5uXNaXXTzlDcDzv+icKXzsl67B3os240y8Itf0/HuxfYDiNf5AMuFpz9+jVKnnRl7hy6ThT5oZCcJIwHWo8AIdzyVsmvCbHu/1YZFXN1x76VZpscSBXnnuq/Ww49W9QEb5d+D5q8lvJafUk4P/+MCfNDMSBHHwL4rrUNPqXiDT0nkANe0q0KvgXw1ClHsM/ucU2LhdoX6H1UZi+6QSvhu4zp9Jw7YPMxKEUdSRH+05pDc6zx5XFGkS129814HvkrZdql0q/8mSCyK5Z0E3yj+v1JOV//jAnzQzEgSR6MijL3kO5mzK3xl8WWPFd1O9fboJ/kSE6Xgv306qfZymUk/KBH9W/uMCK39mJAgj0dG2ize99rKUNaIYRJ1/lsmSm5R6dpnwBWTSd7keYG6y80VeurGbl63zZz04LnDwZ0aCIBIdKfVrL7WX9ZnnGJS/PeG7WG2EiCKBaiPqqs4fkM3dtkyVOs6FAGnPP/BY+Y8jHPyZkSDqMOGbR3onr8HaPqqnfze2DyCTvp2+By9b6uk6qPjJGopR242K6R4O/sxIIJV/75ZEqrfPgG2fZCOX7t7XCy/egka8+K0oZktnJ97YRJV6cqXPeMHBnxkJwj4p//ROXoPxtydKUvlXu9y/V3Hja5/V8XM8I+GrKnsch1ByHa70GTP402ZGgk49/zzSe/gOyvP3sFoPu97FqxfUe64FUap9c9nn4D9u8KfNWLn74Clc/t9u1+0IBk0YRf1R/s7gPf+Jkkz46l28ulT+3WC+Z7PDZcV3Odk7ZnDwZ6w8enwRS7UAR+abuyAKIXDPoVMQojO/uReCsD+2j+PITbyBAXr+votVw/bpNuHbDeZ7NoN9xe//RifMcMOfNmNlsRqk/je5//A8rr/5Htxz6PS6jaceRn2zJZT6H9QiL638B2D7eKngb9g+Hiv/cYODP2NFBf0lS/A/sSg3DT8Zbx6+HixVA8xU+lOfoNRvp/vm9otKrPyV7TMo5W96/hX2/MeOnj5tInoLET1ARBER7TWO+0T0N0T0HSJ6iIhuNO67Oj5+gIg+QFxYPJQsVhvy/1qj+b74mO2qYK1YqDb6FvxVSeOglP9kyUU9jLAc7+ZV6bLUsxuISE8AZrCveO6G3sKRaabXT3s/gDcBuDNz/C0AykKIKwBcDeDtRHRRfN+HAdwAYE/877oex8CsAa1sn4VVeWyh2jwxrOV4ZivNvXq6QQX9QXn+SumfXpa/v/VM+ALJ+/a95P1PlNwNvYsX00xPUkoI8RAA26pAAWCKiDwAEwDqABaIaAeAWSHE3fHzPgrgDQBu62UcTP/Ryt8S/JP71i/4L6w2MNOv4O8O3vMHoCup1tP2AZKKH1P5v+sVe1CL+/0w48FaLfL6FIDXAzgKYBLArwshTsfW0GHjcYcB7FyjMTA9sFTLV/6trgrWgjASWK6HmJ3ok+0zLMp/JQ7+65jwBZL3bwb/qy/csq5jYAZP228TEd0B4DzLXTcJIT6T87QXAAgBnA9gM4Cvxuexfdty6wWJ6AZIiwi7d+9uN1SmjyQBvlndK7tnYXV9lL9KOvdP+RMcsl6xrgsq2J9eksHf3ElrPVALvUrs8Y81bYO/EOIVXZz3JwF8QQjRAHCciO4CsBfAVwHsMh63C8CRFq99M4CbAWDv3r3rV1TOtPb811n5q8mmbwlfZ7A17Tr4r9RR9px1rzpKlD/XWowza/UNeALAy0gyBeAaAA8LIY4CWCSia+Iqn7cCyLt6YAbIQgtfXyn+9Ur4qtfpV8LXdWhglg+Q2D5nluvrnuwFYK32YcaPXks930hEhwFcC+DzRHR7fNcHAUxDVgN9HcBHhBD3x/e9A8AtAA4AOAhO9g4dQgjt+av/Tdbb81fVRbN9U/6DDf6TKuG7Ul/3ZC+QNLTzubpnrOm12udWALdaji9BlnvanrMPwOW9vC6ztizXQ6jODa2rfXoP/kfOruK6992JT7z9Wjxrx6z1Mer1bBuyd4PnDonyX2lg95bJdX999d7Z8x9v+NNnmjCtnlaefz8Svo8eX8JCNcDDxxZajEclfPvp+Q8u+CurJ4zEgG0f9vzHGQ7+TBMq2J4zXbb6+mpyWKoHiKLe8vCnl2WLiFNL+d1Dk4Rv/xZ5Daq1A5DYPkD3G7n0Anv+DMDBn7Gggv/OuQqWakGqe2c9iFBtRJib9CEEsGjJCXSCCvqnW7SO7rvyd2mgyt+s61/vGn/AXufPjB/86TNNKGW/Y9MEhJA5gOx9O+cmUre7RQX9MystlP9qAxO+27dg5TnOQD1/s65/vWv8AcPz54TvWMOfPtOEUto75irx7eYcQBL8e1P+Kui3sn0Wq0HfVvcCg1f+jkO6mVtloMqfPf9xhoM/00SrAK/8952b5X29Jn1ttk8UCcwb512s9a+vDzB4zx8AJktyMhtEqSd7/gzAwZ+xsBS3bD7fEvz7rfxV0DeD/6e+eRgv/n/+Wfe7X1jtXy9/ANg0UerbgrFuUUGfgz8zKNaqsRszwixWAxAB22fL8e1EhSulr4J/r6t8VdA/ZQT/R44tYrEW4Oj8Ki7ZNo3FagNzk6WeXsfk/3rtM1ENBtvBUtk+g0j4cp0/A7DyZywsVgNMlz1ttViV/+b+KH8V9OdXG2iEMiAfm6+m/l/o4y5eALB1uqwnr0GhbJ9B1PnrFb7s+Y81HPyZJhaqDcxWfB1wbZ7/+X2o9mmEEeZXGzhnWl5hqOTvsYVq6v/FaqNvq3uHhaGwfbjaZ6zhT59pQu2Xq5T/krGV40JsCW2ZLKHsOXq1bzeoYP+0c6fk7XhnK638F9ZG+Q8Dyu5Zzy0cFVznzwAc/BkLi3GwnSq5IMraPg1Mlzw4DmGm4vek/FWwf9q50wCAU8s1RJHAU3HQf2q+imojRD2IBp6g7TfDoPzZ8x9v+NNnmlisNTBd9kBEmC57adtnNdAWzOyEpztudsOpuLXDnnNnAMjk76nlOoK4ZcTR+ap+7X519BwWlPIfZMKXlf94w58+04RU/nGAr/ipip7FakNbMDOZ+zpFVfrsiZX/6eW6Vv0OAU8tVPve12dYSGwfbuzGDAYO/kwTS4bHPlPx9DaKQLzaVk8MXk/VPir4X6psn6W69vufvn0GxxYM5d/HFb7DwCBtH48Tvgw4+DMWTOU/kwnwC4byz14VdIpa3bt1qoS5SR+nl+s4Giv/q3bN4cRiTSeFN5rynxyo7SO/9uz5jzf86TMpqo0Q9TDSAX667GGxlu7tY14V9Kr8N0348FwHWyZLOL1Sx1PzVbgO4fKds4gEcOjEsn6tjYSyewbR2I2rfRiAgz+TIds+WVb0pJV/kvD1e+rtc3qljq1TcuXulqkSTi/VcWyhim3TZb2I7NGnFuVrbTDlr22f0gD6+bvs+TPc3oHJoPbstXn+Qoi08i97qAUR6kHUVXvg00t1bDGC/+OnVuA6hO2bKtg+KzuKfjcO/htN+W+dLoGof1tTdgIrfwbg4M9kUHX7M2Xl+SfKf7URIoxEkvCd8PVztsardDvh9HIdF26Ve9hunS7hm0+cRSgELt02hfPi4P/o8SU4BEyVNtaf6msu34ELt07h3JnKur+2Q9zPn2Hbh8mgAv20ofzrYYRqI9Q1/WYyGEDXq3xPLdexdTpR/mdWZLXPjk0T2DJVQsl1dJ+hQbdg7jclz8FzLpgbyGuz8mcADv5MBq38jeAvjweW+/zUczohigTOrNSxOe7WuXmyhDASWKoF2D5bARHh3Lir6Ear9Bk07PkzAAd/JkOyojat7pdqgS7r1Alfpfy7WOW7UG0gjIT2/NUVAACct0kGfWX9bLSmboPG4/YODDj4Mxmaqn3KibpfsFQCqfs6RbVyTmyfJGegkr3nbaqkXo/pD7rOnz3/sYY/fSaFCv5T5bjOP2X7pK8K1Krbbmr9z8TBXwV9VfIJyI3jAUP5s+3TV9jzZ4Aegz8R/TERPUxE9xPRrUQ0Z9x3IxEdIKJHiOjVxvGrieg78X0fICI2HoeIxWoDE76rA0Pi+Td0Tf9sRvl3s8pXK3+j1FNxXkb5b7SmboPmnOkyZioeK/8xp9dP/0sALhdCXAnguwBuBAAiugzA9QCeDeA6AB8iIrWU8cMAbgCwJ/53XY9jYPrIUi3dO3/W2M0rsYTksely99U+p7XyTwf/2YqnWx4o+4dtn/7ylr278OXf/CFW/mNOT5++EOKLQgj1zb8HwK7459cD+LgQoiaEeAzAAQAvIKIdAGaFEHcLIQSAjwJ4Qy9jYPqLuYgLSFf7LFQb8F3SG5C4DmGm7HW1yjcb/Cu+i8mSq9U+YCh/Tvj2Fd91BrK+gBku+impfh7AJ+Kfd0JOBorD8bFG/HP2+Jrxi3/zdTx+amUtX2JDceTsKp62fUbfVur+g185gCASmKn4MJ26mYqHf/zmYdx14GRHr3NyqYbJkptqabxlqqTVPpDYP6z8Gab/tP1WEdEdAM6z3HWTEOIz8WNuAhAA+Hv1NMvjRYvjea99A6RFhN27d7cbqpXdW6bY2+yAPdun8epnJx+35zp458v34NHjss3C1RduST3+HS99Gu4+2FngV6+TXeT0G698ut7PFwB2bZ7AO1++B6+5fEfH52cYpjUk3ZceTkD0NgC/DODlQoiV+NiNACCE+IP49u0A3gPgewC+IoR4Znz8JwD8sBDi7e1eZ+/evWLfvn09jZVhGGbcIKJvCCH2Zo/3Wu1zHYDfBvA6FfhjPgvgeiIqE9HFkIndrwkhjgJYJKJr4iqftwL4TC9jYBiGYTqnVzP1LwCUAXwp9oHvEUL8shDiASL6JIAHIe2gXxFChPFz3gHgrwFMALgt/scwDMOsIz0FfyHE01rc914A77Uc3wfg8l5el2EYhukNzoQyDMOMIRz8GYZhxhAO/gzDMGMIB3+GYZgxhIM/wzDMGNLzIq/1gohOAHi8y6efA6DzZajDwSiPHeDxD5JRHjsw2uMfprFfKITYlj04MsG/F4hon22F2ygwymMHePyDZJTHDoz2+Edh7Gz7MAzDjCEc/BmGYcaQcQn+Nw96AD0wymMHePyDZJTHDoz2+Id+7GPh+TMMwzBpxkX5MwzDMAYbOvgT0XXxBvIHiOi/DHo87SCiC4joK0T0EBE9QETvjI9vIaIvEdGj8f+bBz3WPIjIJaJvEdHn4tujNPY5IvoUET0cfwbXjsr4iejX47+Z/UT0MSKqDPPYieiviOg4Ee03juWOl4hujL/HjxDRqwcz6oSc8f9x/LdzPxHdSkRzxn1DNX5gAwf/eMP4DwJ4DYDLAPxEvLH8MBMA+E0hxLMAXAPgV+Ix/xcAXxZC7AHw5fj2sPJOAA8Zt0dp7O8H8IV4s6GrIN/H0I+fiHYC+DUAe4UQlwNwAVyP4R77XwO4LnPMOt74O3A9gGfHz/lQ/P0eJH+N5vF/CcDlQogrAXwXwI3A0I5/4wZ/AC8AcEAIcUgIUQfwcciN5YcWIcRRIcQ3458XIYPPTshx/038sL/BkG56T0S7APwIgFuMw6My9lkALwHwlwAghKgLIc5iRMYP2Z59gog8AJMAjmCIxy6EuBPA6czhvPG+HsDHhRA1IcRjAA5Afr8Hhm38QogvCiGC+OY9AHbFPw/d+IGNHfx3Avi+cXvNN4vvJ0R0EYDnArgXwPZ4FzTE/587uJG15H0A3g0gMo6NytgvAXACwEdi2+oWIprCCIxfCPEkgD8B8ASAowDmhRBfxAiMPUPeeEfxu/zzSDaqGsrxb+Tg39Fm8cMEEU0D+AcA7xJCLAx6PEUgoh8FcFwI8Y1Bj6VLPADPA/BhIcRzASxjuGySXGJv/PUALgZwPoApIvrpwY6qr4zUd5mIboK0cP9eHbI8bODj38jB/zCAC4zbuyAvhYcaIvIhA//fCyH+MT78FBHtiO/fAeD4oMbXgh8A8Doi+h6kxfYyIvo7jMbYAfn3clgIcW98+1OQk8EojP8VAB4TQpwQQjQA/COAF2E0xm6SN96R+S4T0dsA/CiAnxJJHf1Qjn8jB/+vA9hDRBcTUQky4fLZAY+pJfGm9n8J4CEhxJ8ad30WwNvin9+GIdz0XghxoxBilxDiIsjf9T8LIX4aIzB2ABBCHAPwfSJ6Rnzo5ZB7UI/C+J8AcA0RTcZ/Qy+HzBeNwthN8sb7WQDXE1GZiC4GsAfA1wYwvpYQ0XUAfhvA64QQK8Zdwzl+IcSG/QfgtZBZ94MAbhr0eAqM98WQl4P3A7gv/vdaAFshqx8ejf/fMuixtnkfPwzgc/HPIzN2AM8BsC/+/X8awOZRGT+A3wPwMID9AP4WQHmYxw7gY5D5iQakMv6FVuMFcFP8PX4EwGuGdPwHIL199d39/4Z1/EIIXuHLMAwzjmxk24dhGIbJgYM/wzDMGMLBn2EYZgzh4M8wDDOGcPBnGIYZQzj4MwzDjCEc/BmGYcYQDv4MwzBjyP8PhwAp+NojEf4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def play_episode(env, agent, max_episode_steps=None, mode=None, render=False):\n",
    "    observation, reward, done = env.reset(), 0., False\n",
    "    agent.reset(mode=mode)\n",
    "    episode_reward, elapsed_steps = 0., 0\n",
    "    while True:\n",
    "        action = agent.step(observation, reward, done)\n",
    "        if render:\n",
    "            env.render()\n",
    "        if done:\n",
    "            break\n",
    "        observation, reward, done, _ = env.step(action)\n",
    "        episode_reward += reward\n",
    "        elapsed_steps += 1\n",
    "        if max_episode_steps and elapsed_steps >= max_episode_steps:\n",
    "            break\n",
    "    agent.close()\n",
    "    return episode_reward, elapsed_steps\n",
    "\n",
    "\n",
    "logging.info('==== train ====')\n",
    "episode_rewards = []\n",
    "for episode in itertools.count():\n",
    "    episode_reward, elapsed_steps = play_episode(env.unwrapped, agent,\n",
    "            max_episode_steps=env._max_episode_steps, mode='train')\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('train episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "    if np.mean(episode_rewards[-10:]) > -110:\n",
    "        break\n",
    "plt.plot(episode_rewards)\n",
    "\n",
    "\n",
    "logging.info('==== test ====')\n",
    "episode_rewards = []\n",
    "for episode in range(100):\n",
    "    episode_reward, elapsed_steps = play_episode(env, agent)\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('test episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "logging.info('average episode reward = %.2f ± %.2f',\n",
    "        np.mean(episode_rewards), np.std(episode_rewards))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
